Process the result of the product classifier.

* Prompt tweaks

Author: akatsoulas

kitsune/llm/questions/classifiers.py

@@ -12,10 +12,7 @@
     topic_prompt,
 )
 from kitsune.llm.utils import get_llm
-
-# TODO:
-# from kitsune.products.utils import get_products, get_taxonomy
-from kitsune.products.utils import get_taxonomy
+from kitsune.products.utils import get_products, get_taxonomy
 
 DEFAULT_LLM_MODEL = "gemini-2.5-flash-preview-04-17"
 HIGH_CONFIDENCE_THRESHOLD = 75
@@ -66,8 +63,7 @@ def handle_spam(payload: dict[str, Any], spam_result: dict[str, Any]) -> dict[st
         if not ((action == ModerationAction.SPAM) and spam_result.get("maybe_misclassified")):
             return {"action": action, "product_result": {}}
 
-        # TODO:
-        # payload["products"] = get_products(output_format="JSON")
+        payload["products"] = get_products(output_format="JSON")
         product_result = product_classification_chain.invoke(payload)
         new_product = product_result.get("product")
 

kitsune/llm/questions/prompt.py

@@ -29,7 +29,8 @@
    - `0` = Extremely uncertain.
    - `100` = Completely certain.
 4. Provide a concise explanation supporting your decision.
-5. **Determine if the question may have been classified under the wrong product:** This should be true if and only if the question represents a legitimate support request, **and** the **sole** reason for classifying the question as spam is because it's clearly unrelated to Mozilla's "{product}" product.
+5. **Determine if question was misclassified due to wrong product:** True only if this is a legitimate Mozilla support request that 
+doesn't relate to "{product}" but clearly relates to another Mozilla product.
 
 # Response format
 {format_instructions}
@@ -147,7 +148,10 @@
         ResponseSchema(
             name="maybe_misclassified",
             type="bool",
-            description="A boolean that when true indicates that the question may be classified under the wrong product",
+            description=(
+                "True if this appears to be a legitimate Mozilla support request"
+                " that was flagged as spam solely due to incorrect product categorization."
+            ),
         ),
     )
 )

kitsune/products/utils.py

@@ -1,7 +1,7 @@
 import json
 
-from django.db.models import Prefetch, Q
 import yaml
+from django.db.models import Prefetch, Q
 
 from kitsune.products.models import Product, Topic
 

kitsune/questions/utils.py

@@ -176,41 +176,71 @@ def process_classification_result(
 ) -> None:
     """
     Process the classification result from the LLM and take moderation action.
+    Handles spam, flag review, and updates to product and topic if suggested by the classifier.
     """
     sumo_bot = Profile.get_sumo_bot()
     action = result.get("action")
-    match action:
-        case ModerationAction.SPAM:
-            question.mark_as_spam(sumo_bot)
-        case ModerationAction.FLAG_REVIEW:
+
+    if action == ModerationAction.SPAM:
+        question.mark_as_spam(sumo_bot)
+        return
+    elif action == ModerationAction.FLAG_REVIEW:
+        flag_question(
+            question,
+            by_user=sumo_bot,
+            notes=(
+                "LLM flagged for manual review, for the following reason:\n"
+                f"{result.get('spam_result', {}).get('reason', '')}"
+            ),
+            reason=FlaggedObject.REASON_SPAM,
+        )
+        return
+
+    product_result = result.get("product_result", {})
+    topic_result = result.get("topic_result", {})
+    new_product_title = product_result.get("product")
+    new_topic_title = topic_result.get("topic")
+
+    update_kwargs = {}
+
+    if (
+        new_product_title
+        and hasattr(question, "product")
+        and getattr(question.product, "title", None) != new_product_title
+    ):
+        from kitsune.products.models import Product
+
+        try:
+            new_product = Product.objects.get(title=new_product_title)
+        except Product.DoesNotExist:
+            log.warning(
+                f"LLM suggested product '{new_product_title}' does not exist. Skipping product update."
+            )
+        else:
+            update_kwargs["product"] = new_product
+
+    if new_topic_title:
+        try:
+            topic = Topic.active.get(title=new_topic_title, visible=True)
+        except (Topic.DoesNotExist, Topic.MultipleObjectsReturned):
+            log.warning(
+                f"LLM suggested topic '{new_topic_title}' is invalid. Skipping topic update."
+            )
+        else:
+            update_kwargs["topic"] = topic
+
+    if update_kwargs:
+        question.save(**update_kwargs)
+        question.clear_cached_tags()
+        question.auto_tag()
+
+        if update_kwargs.get("topic"):
             flag_question(
                 question,
                 by_user=sumo_bot,
                 notes=(
-                    "LLM flagged for manual review, for the following reason:\n"
-                    f"{result['spam_result']['reason']}"
+                    f"LLM classified as {topic.title}, for the following reason:\n"
+                    f"{topic_result.get('reason', '')}"
                 ),
-                reason=FlaggedObject.REASON_SPAM,
+                status=FlaggedObject.FLAG_ACCEPTED,
             )
-        case _:
-            if topic_title := result["topic_result"].get("topic"):
-                try:
-                    topic = Topic.active.get(title=topic_title, visible=True)
-                except (Topic.DoesNotExist, Topic.MultipleObjectsReturned):
-                    return
-                else:
-                    flag_question(
-                        question,
-                        by_user=sumo_bot,
-                        notes=(
-                            "LLM classified as {topic.title}, for the following reason:\n"
-                            f"{result['topic_result']['reason']}"
-                        ),
-                        status=FlaggedObject.FLAG_ACCEPTED,
-                    )
-                    if question.topic:
-                        question.tags.remove(question.topic.slug)
-                    question.topic = topic
-                    question.save()
-                    question.tags.add(topic.slug)
-                    question.clear_cached_tags()